package com.java.blog.webmagic.processor;

import java.util.List;

import org.apache.commons.lang3.StringUtils;

import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.selector.Selectable;

public class QsbkRepoPageProcessor implements PageProcessor {
	
	private static final String UserAgent = "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.101 Safari/537.36";

	@Override
	public Site getSite() {
		// 部分一：抓取网站的相关配置，包括编码、抓取间隔、重试次数等
		return Site.me().setUserAgent(UserAgent).setRetryTimes(3).setSleepTime(1000);
	}

	@Override
	// process是定制爬虫逻辑的核心接口，在这里编写抽取逻辑
	public void process(Page page) {
		// 部分二：定义如何抽取页面信息，并保存下来
		List<Selectable> items = page.getHtml().xpath("//div[@class='article block untagged mb15']").nodes();
		JSONArray qiushi = new JSONArray();
		for (Selectable selectable : items) {
			// 获取名字，内容，图片
			String name = selectable.xpath("//div[@class='author clearfix']/a[2]/h2/text()").get();
			String content = selectable.xpath("//div[@class='content']/text()").get();
			String image = selectable.xpath("//div[@class='thumb']/a/img/@src").get();
			// 装入对象
			JSONObject item = new JSONObject();
			item.put("name", name);
			item.put("content", content);
			if (StringUtils.isNotEmpty(image)) {
				item.put("image", image);
			}
			qiushi.add(item);
		}
		page.putField("qiushi", qiushi);
	}
}